{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-d7d514ac5c52>:5: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Program\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Program\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting C:/Users/Administrator/workspace/homework6/courseware\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Program\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting C:/Users/Administrator/workspace/homework6/courseware\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Program\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting C:/Users/Administrator/workspace/homework6/courseware\\t10k-images-idx3-ubyte.gz\n",
      "Extracting C:/Users/Administrator/workspace/homework6/courseware\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Program\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# 我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载\n",
    "# 这里将data_dir改为适合你的运行环境的目录\n",
    "# import data\n",
    "data_dir = 'C:/Users/Administrator/workspace/homework6/courseware'\n",
    "mnist = input_data.read_data_sets(data_dir,one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 一个非常非常简陋的模型\n",
    "# Create the model\n",
    "in_units = 784 # 输入节点数\n",
    "h1_units = 1568 # 隐含层的输出节点数 1568:0.9806;   400:0.954;300时0.9538；250:0.9554;200:0.9552;100时0.9551；50：0.954\n",
    "\n",
    "h2_units = 50 # 隐含层2的输入节点数，这层不能多\n",
    "\n",
    "# 输入\n",
    "x = tf.placeholder(tf.float32,[None,in_units])\n",
    "\n",
    "# 隐含层1\n",
    "W1 = tf.Variable(tf.truncated_normal([in_units,h1_units],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([h1_units]))\n",
    "\n",
    "#h1=tf.nn.sigmoid(tf.matmul(x,W1)+b1)\n",
    "#h1=tf.nn.tanh(tf.matmul(x,W1)+b1)\n",
    "#h1=tf.nn.relu(tf.matmul(x,W1)+b1)\n",
    "h1=tf.nn.relu6(tf.matmul(x,W1)+b1)\n",
    "#h1=tf.nn.elu(tf.matmul(x,W1)+b1)\n",
    "#h1=tf.nn.softsign(tf.matmul(x,W1)+b1)\n",
    "\n",
    "# 隐含层2\n",
    "W2 = tf.Variable(tf.truncated_normal([h1_units,h2_units],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([h2_units]))\n",
    "\n",
    "#h2=tf.nn.sigmoid(tf.matmul(h1,W2)+b2) # 0.9512\n",
    "#h2 = tf.nn.softmax(tf.matmul(h1,W2) + b2) # 只有0.7114\n",
    "#h2 = tf.nn.tanh(tf.matmul(h1,W2) + b2) #0.9729\n",
    "#h2 = tf.nn.relu(tf.matmul(h1,W2) + b2) # 0.9756\n",
    "h2 = tf.nn.relu6(tf.matmul(h1,W2) + b2) # 0.9794\n",
    "#h2 = tf.nn.elu(tf.matmul(h1,W2) + b2) # 0.9728\n",
    "#h2 = tf.nn.softsign(tf.matmul(h1,W2) + b2)\n",
    "\n",
    "\n",
    "# 输出层\n",
    "Wy = tf.Variable(tf.zeros([h2_units,10]))\n",
    "by = tf.Variable(tf.zeros([10]))\n",
    "# 使用softmax作为多分类问题的激活函数\n",
    "y = tf.nn.softmax(tf.matmul(h2,Wy) + by)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义我们的ground truth占位符\n",
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32,[None,10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 接下来我们计算交叉熵，注意softmax_cross_entropy_with_logits的logits参数是未经激活的wx+b\n",
    "#cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=y))\n",
    "# L1、L2\n",
    "#w = tf.Variable(tf.random_normal([h2_units,10]),dtype=tf.float32)\n",
    "w = tf.Variable(tf.truncated_normal([h2_units,10],stddev=0.1)) # 不同的初始化: 0.9741\n",
    "vlambda = 0.0055 # 0.005 0.979;0.001 0.9773;0.01 0.9778\n",
    "\n",
    "#cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_*tf.log(y),reduction_indices=[0])) # 0.9199\n",
    "cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_*tf.log(y) + tf.contrib.layers.l2_regularizer(vlambda)(w),reduction_indices=[1])) # 0.9725\n",
    "#cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_*tf.log(y) + tf.contrib.layers.l1_regularizer(vlambda)(w),reduction_indices=[1])) # 0.9739\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 生成一个训练step\n",
    "learing_rate = 0.55  # 0.555:0.9725；0.55:0.9758; 0.5：0.9739; 0.1:0.9639; 0.03:0.9349;0.7:0.098\n",
    "\n",
    "train_step = tf.train.GradientDescentOptimizer(learing_rate).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。然后我们运行3k个step（5 epochs 5训练次数），对权重进行优化\n",
    "for _ in range(3000):\n",
    "    batch_xs,batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    \n",
    "    #correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))\n",
    "    #accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "    #print(\"Loss：{0}；accuracy：{1}\",str(cross_entropy),str(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9802\n"
     ]
    }
   ],
   "source": [
    "# 验证我们模型在测试数据上的准确率\n",
    "# Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    " - 隐层神经元数量\n",
    " - 学习率\n",
    " - 正则化惩罚因子\n",
    " - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
